require 'iconv'
class Dict
	#num
	NUM_MAP = { "一" => "1", "二" => "2", "三" => "3", "四" => "4", "五" => "5", "六" => "6", "七" => "7", "八" => "8", "九" => "9", "十" => "10", "零" => "0", '百' => '100', "千" => "1000" }
	ZH_NUM_MAP = { "壹" => "1", "贰" => "2", "参" => "3", "肆" => "4", "伍" => "5", "陆" => "6", "柒" => "7", "捌" => "8", "玖" => "9", "拾" => "10", "零" => "0", "佰" => "100", "仟" => "1000", "两" => "2" }
	ZH_NUM_EN_MAP = {"１" => "1", "２" => "2", "３" => "3", "４" => "4", "５" => "5", "６" => "6", "７" => "7", "８" => "8", "９" => "9", "０" => "0"}
	
	#dict files
	STOP_WORD_FILE  = "dict/stop_dicts.txt"
	KEY_WORD_FILE   = "dict/keyword_dicts.txt"
	REG_WORD_FILE   = "dict/regexp_dicts.txt"
	BRAND_WORD_FILE = "dict/brand_dicts.txt"
	#init dict array
	STOP_WORDS   = Array.new 
	KEY_WORDS    = Array.new
	REG_WORDS    = Array.new
	BRAND_WORDS  = Array.new
	
	NICK_WORDS   = ["品牌", "旗舰店", "服饰", "专营店"]
	ADDIT_WORDS  = ["包含",  "不含","不送", "不配", "含","送", "配"]
	INVERT_WORDS = ["搭", "褶", "样", "色", "分", "步", "粒", "彩", "穿", "字", "足", "家", "只", "合", "面", "用", "季"]
	PUNCTUAION_WORDS  = {"》" => "《", "）" => "（", ")" => "(", ">" => "<", "]" => "[", "}" => "{"}
	
	RUBBISH_WORDS_REG = /( |，|@|!|#|＆|:|&|\-|_|\*|;|~|!|<|>|、|█|■|●|【|】|★|\[|\]|。|◆|＠|\^|◎|『|』|（|）|？|\?|！|\|．|〗|〖|◤|◢|﹏|,|☆|\?|\|)/
	BRAND_FILTER_REG  = /(:|·|＆|【|】)/i
	
	RUBBISH_WORDS     = ".,，·:_@!|#&\-*;~!/、[]。『』（）！．《》）（ 　	"
	
	load_dict = Proc.new do |file, dict, sort|
		File.open(file, "r") do |file| 
			while line = file.gets
				line = line.rstrip
				next if line.length == 0 || line[0,1] == "#" || dict.include?(line)
				dict << line
			end
		end
 		dict.sort!{|x, y| y.length <=> x.length } if !sort.nil? && sort == true
	end
	#load dict
	puts "--- loading stop dict -- #{Time.now.strftime("%Y-%m-%d %H:%M:%S")} -----"
  load_dict.call(STOP_WORD_FILE, STOP_WORDS, true)	#stop dicts
  puts "--- loaded stop dict (#{STOP_WORDS.size})---------------"
  
  puts "----loading keyword dict -- #{Time.now.strftime("%Y-%m-%d %H:%M:%S")} -----"
	load_dict.call(KEY_WORD_FILE, KEY_WORDS, true )  #keywords
	puts "----loaded keyword dict (#{KEY_WORDS.size}) ---------"
	
	puts "----loading regexp word dict -- #{Time.now.strftime("%Y-%m-%d %H:%M:%S")} -----"
	load_dict.call(REG_WORD_FILE, REG_WORDS)  #reg keywords
	puts "----loaded regexp word dict (#{REG_WORDS.size}) ---------"
	
	puts "----loading brand dict -- #{Time.now.strftime("%Y-%m-%d %H:%M:%S")} -----"
	load_dict.call(BRAND_WORD_FILE, BRAND_WORDS)  #brands
	puts "----loaded brand dict (#{BRAND_WORDS.size}) ---------"
end
